# -*- coding: utf-8 -*-
"""
Created on 2024-08-17 17:03:20
---------
@summary:
---------
@author: zb
"""

import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import random
import time
import json
import uuid
from util import MyUtils
import feapder
import datetime
from curl_cffi import requests
from urllib.parse import quote
from feapder.utils import tools
from feapder.utils.log import log
from feapder.db.redisdb import RedisDB
from feapder.db.mysqldb import MysqlDB
from curl_cffi.requests import Response
from ratelimit import sleep_and_retry, limits
from spiders.公示系统.item.newitem import ErrorItem, YearItem, UpdataReportNumberItem, PhoneItem
from userPool.phonePool import user_pool

# TODO  1111111111111111111===================================================================
p_count_name = "phone" + ":count"
p_cache_total = "total"
p_cache_success = "success"
p_cache_error = "error"


# TODO 1111111111111111111111===================================================================

class 官渡区(feapder.TaskSpider):
    # 自定义数据库，若项目中有setting.py文件，此自定义可删除
    db_name = "qierge_53"
    _rdc = None
    _db = None

    # TODO 222222222222222222222222222222=================================================================
    p_url = "https://capi.tianyancha.com/cloud-tempest/web/searchCompanyV3?_={}"
    # 移动
    p_cmc = ['1340', '1341', '1342', '1343', '1344', '1345', '1346', '1347', '1348', '135', '136', '137', '138', '139',
             '144', '147', '148', '150', '151', '152', '157', '158', '159', '165', '1703', '1705', '1706', '172', '178',
             '182', '183', '184', '187', '188', '195', '197', '198']
    # 联通
    p_cuc = ['130', '131', '132', '140', '145', '146', '155', '156', '166', '167', '1704', '1707', '1708', '1709',
             '171',
             '175', '176', '185', '186', '196']
    # 电信
    p_ctc = ['133', '1349', '141', '149', '153', '161', '162', '1700', '1701', '1702', '173', '1740', '1741', '1742',
             '1743', '1744', '1745', '177', '180', '181', '189', '190', '191', '193', '199']

    # TODO 222222222222222222222222222222====================================================================

    __custom_setting__ = dict(
        REDISDB_IP_PORTS="localhost:6379",
        # REDISDB_IP_PORTS = "139.9.196.26:6379"
        REDISDB_USER_PASS="python",
        REDISDB_DB=9,
        MYSQL_IP="rm-bp152ng910khxljk85o.mysql.rds.aliyuncs.com",
        MYSQL_PORT=3306,
        MYSQL_DB=db_name,
        MYSQL_USER_NAME="dev",
        MYSQL_USER_PASS="3NG1LRYKCVKa8dQZ+HkpcQ",
        LOG_LEVEL="INFO"
    )

    @property
    def db(self):
        if not self._db:
            self._db = MysqlDB()
        return self._db

    @property
    def rdc(self):
        if not self._rdc:
            self._rdc = RedisDB()
        return self._rdc

    @property
    def get_digest(self):
        return uuid.uuid4().hex.upper()

    def start_requests(self, task):
        task_id = task.id
        task_name = task.name
        task_regNumber = task.regNumber
        task_creditCode = task.creditCode
        task_digest = task.digest
        phones = task.phone
        # TODO digest有的为空
        if not task_digest:
            task_digest = self.get_digest
        searchword = task_regNumber or task_creditCode or task_name
        task_city = task.city  # 市
        task_district = task.district  # 区
        provinceId = task.provinceId  # 省编号
        task_regStatus = task.regStatus  # 经营状态
        data = {
            "conditions": json.dumps({
                "excep_tab": "0",
                "ill_tab": "0",
                "area": f"{provinceId}",
                "cStatus": "0",
                "xzxk": "0",
                "xzcf": "0",
                "dydj": "0"
            }, sort_keys=False, separators=(',', ':')),
            # "searchword": util.url_quote(name),
            "searchword": searchword,
            "sourceType": "W"
        }
        url = "https://app.gsxt.gov.cn/gsxt/corp-query-app-search-1.html"
        if searchword:
            yield feapder.Request(
                url=url,
                data=data,
                auto_request=False,
                task_id=task_id,
                task_name=task_name,
                task_regNumber=task_regNumber,
                task_creditCode=task_creditCode,
                task_digest=task_digest,
                task_city=task_city,
                searchword=searchword,
                task_district=task_district,
                task_regStatus=task_regStatus,
                phones=phones
            )
        else:
            self.myLog(task_city)
            self.db.update_smart(table_name, {"reportState": 4}, condition=f"id={task_id}")

    # TODO 33333333333333333333333333333333333333=====================================================================
    @sleep_and_retry
    @limits(calls=4, period=1, raise_on_limit=False)
    def download_midware(self, request):
        self.rdc.hincrby(p_count_name, p_cache_total, 1)
        request.proxies = MyUtils.get_proxy()
        tyc_id = user_pool.get_user().tyc_id
        request.headers = {
            "Accept": "application/json, text/plain, */*",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "close",
            "Content-Length": "136",
            "Content-Type": "application/json",
            "Host": "capi.tianyancha.com",
            "Origin": "https://www.tianyancha.com",
            "Pragma": "no-cache",
            "Referer": "https://www.tianyancha.com/",
            "Sec-Fetch-Dest": "empty",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Site": "same-site",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                          "Chrome/119.0.0.0 Safari/537.36",
            "X-TYCID": tyc_id,
            "sec-ch-ua": "\"Google Chrome\";v=\"119\", \"Chromium\";v=\"119\", \"Not?A_Brand\";v=\"24\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Windows\"",
            "version": "TYC-Web"
        }
        request.url = request.url.format(MyUtils.get_current_timestamp(msec=True))
        '''todo  有好的就赶快走哦前人的经验'''
        return request

    def validate(self, request, response):
        state = response.json.get('state')
        vip = response.json.get('vipMessage')
        message = response.json.get('message')
        if state != "ok" or vip or message:
            self.rdc.hincrby(p_count_name, p_cache_error, 1)
            time.sleep(random.choice(range(5, 10)))
            raise ValueError(response.json)

    # TODO 33333333333333333333333333333333333333333==================================================================
    def download(self, url=None, data=None, params=None) -> Response:
        @tools.retry(10, 5)
        def _fun():
            method = "POST" if data else "GET"
            header = {
                "Host": "app.gsxt.gov.cn",
                "Connection": "close",
                "Accept": "application/json",
                "xweb_xhr": "1",
                "User-Agent": f"{MyUtils.get_UA()} MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090b19)XWEB/11065",
                "Content-Type": "application/x-www-form-urlencoded",
                "Sec-Fetch-Site": "cross-site",
                "Sec-Fetch-Mode": "cors",
                "Sec-Fetch-Dest": "empty",
                "Accept-Encoding": "gzip, deflate, br",
                "Accept-Language": "zh-CN,zh;q=0.9"
            }
            response = requests.request(method=method, url=url, headers=header, params=params, data=data,
                                        proxies=MyUtils.get_proxy(), verify=False)
            if not response or response is None:
                raise ValueError("fun 函数 : 响应为空")
            if response.status_code != 200:
                raise ValueError(f"fun 函数 : 响应状态{response.status_code}")
            if not getattr(response, "json", None) or (response.json() is None):
                raise ValueError("fun 函数 : 获取json失败")
            return response

        response = _fun()
        return response

    def parse(self, request, response):
        # 下载
        res = self.download(url=request.url, data=request.data)
        if not getattr(res, "json", None):
            yield feapder.Request(
                url=request.url,
                data=request.data,
                auto_request=False,
                task_id=request.task_id,
                task_name=request.task_name,
                task_regNumber=request.task_regNumber,
                task_creditCode=request.task_creditCode,
                task_digest=request.task_digest,
                task_city=request.task_city,
                searchword=request.searchword,
                task_district=request.task_district,
                task_regStatus=request.task_regStatus,
                phones=request.phones
            )
            return
        # log.info(f"{request.searchword} : {res.json()}")
        a = res.json()["data"]
        if not a:
            self.rdc.hincrby("thisTask:errorSum", f"{request.task_id}data", 1)
            if self.rdc.hget("thisTask:errorSum", f"{request.task_id}data") >= 5:
                #Todo 三号无结果
                self.db.update_smart(table_name, {"reportState": 6}, condition=f"id={request.task_id}")
                return
            else:
                raise ValueError("基础解析data网络波动")
        result = a["result"]
        # 其他原因可能导致响应成功但是没有结果
        if not result:
            self.rdc.hincrby("thisTask:errorSum", f"{request.task_id}result", 1)
            if self.rdc.hget("thisTask:errorSum", f"{request.task_id}result") >= 5:
                #Todo 三号无结果
                self.db.update_smart(table_name, {"reportState": 6}, condition=f"id={request.task_id}")
                return
            raise ValueError("基础解析result网络波动")
        dataList = result["data"]
        searchword = request.searchword
        # TODO 没查到结果
        if len(dataList) == 0:
            # TODO  信用号存在的情况下 工商号查不到就换信用号
            if request.task_creditCode and searchword == request.task_regNumber:
                data = request.data
                data["searchword"] = request.task_creditCode
                yield feapder.Request(
                    url=request.url,
                    data=data,
                    auto_request=False,
                    task_id=request.task_id,
                    task_name=request.task_name,
                    task_regNumber=request.task_regNumber,
                    task_creditCode=request.task_creditCode,
                    task_digest=request.task_digest,
                    task_city=request.task_city,
                    searchword=request.task_creditCode,
                    task_district=request.task_district,
                    task_regStatus=request.task_regStatus,
                    phones=request.phones
                )
                return
            # TODO 公司名字存在的情况下  注册号查不到就换公司名称查询
            if (request.task_name and searchword == request.task_creditCode) or (
                    searchword == request.task_regNumber and not request.task_creditCode and request.task_name):
                data = request.data
                data["searchword"] = quote(request.task_name)
                yield feapder.Request(
                    url=request.url,
                    data=data,
                    auto_request=False,
                    task_id=request.task_id,
                    task_name=request.task_name,
                    task_regNumber=request.task_regNumber,
                    task_creditCode=request.task_creditCode,
                    task_digest=request.task_digest,
                    task_city=request.task_city,
                    searchword=request.task_name,
                    task_district=request.task_district,
                    task_regStatus=request.task_regStatus,
                    phones=request.phones
                )
                return
            # TODO 如果公司名称都查不到则直接更新任务为完成状态
            if searchword == request.task_name \
                    or (searchword == request.task_creditCode and not request.task_name) \
                    or (searchword == request.task_regNumber and not request.task_creditCode and not request.task_name):
                self.myLog(request.task_city)
                self.db.update_smart(table_name, {"reportState": 6}, condition=f"id={request.task_id}")  # 300 三号都查不到
                return
            return  # 保险存在的
        re_data = None  # 更新基本信息
        # TODO 如果是公司名称来查询的话 re——data做特需处理
        if searchword == request.task_name:
            for data in dataList:
                regOrg: str = data["regOrg"]
                if data["uniscId"] == request.task_creditCode or data[
                    "regNo"] == request.task_regNumber:  # 切记不能等于名字因为有名字相同的
                    re_data = data
                    # TODO 待优化
                elif regOrg.startswith(f"{request.task_city}{request.task_district}"):
                    re_data = data
        else:
            re_data = dataList[0]
        # TODO 如果公司名称查到了，但不是想要的结果也直接更新任务为完成状态
        if not re_data:  # 没有结果
            self.myLog(request.task_city)
            self.db.update_smart(table_name, {"reportState": 6}, condition=f"id={request.task_id}")
            return
        # TODO 查到了更新基本信息
        self.updata_base(re_data, request.task_id, request.task_regStatus, request.task_digest)
        # TODO 不是存续状态则直接更新完成并且结束任取任务 或者变更为公司
        if request.task_regStatus == 1 and (int(re_data["corpStatus"]) != 1 or int(re_data["entType"]) != 9500):
            self.myLog(request.task_city)
            self.db.update_smart(table_name, {"reportState": 3}, condition=f"id={request.task_id}")
            return
        if request.task_regStatus == 4 and (int(re_data["corpStatus"]) != 1 or int(re_data["entType"]) != 9500):
            self.myLog(request.task_city)
            self.db.update_smart(table_name, {"reportState": 5}, condition=f"id={request.task_id}")
            return
        # TODO 拿到了正确的结果，并且最新状态是存续状态
        # 经营异常
        error_year_params = {"nodeNum": re_data["nodeNum"], "entType": "16", "sourceType": "W"}
        yield feapder.Request(
            url=f"https://app.gsxt.gov.cn/gsxt/corp-query-entprise-info-indBusExcep-{re_data['pripid']}.html",
            auto_request=False,
            params=error_year_params,
            callback=self.error_parase,
            task_digest=request.task_digest,
            task_name=request.task_name
        )
        # 年报
        yield feapder.Request(
            url=f"https://app.gsxt.gov.cn/gsxt/corp-query-entprise-info-anCheYearInfo-{re_data['pripid']}.html",
            auto_request=False,
            params=error_year_params,
            callback=self.year_parase,
            task_id=request.task_id,
            task_name=request.task_name,
            task_digest=request.task_digest,
            estDate=re_data["estDate"],
            task_city=request.task_city
        )

        # TODO 4444444444444444444444444444444444444444444444444444===================================================================

        p_phones = request.phones
        p_task_id = request.task_id
        if p_phones and p_phones != '-':
            p_phones = p_phones.strip().split(';')
            length = len(p_phones)
            number = 0  # 统计过滤次数

            for phone, priori in zip(p_phones, range(len(p_phones))):
                phone: str = phone.replace('\t', '').strip()
                if phone.startswith('0') or "-" in phone or len(phone) != 11:  # 排除座机
                    number += 1
                    continue

                data = {
                    "word": phone,
                    "sortType": "0",
                    "pageSize": 20,
                    "pageNum": 1,
                    "referer": "search",
                    "key": phone,
                    "sessionNo": "1697548110.14294740",
                }
                yield feapder.Request(
                    url=self.p_url,
                    json=data,
                    task_id=p_task_id,
                    phone=phone,
                    priori=priori,
                    digest=request.task_digest,
                    callback=self.phone_parse
                )
            if number == length:
                self.myLog(request.task_city)
                self.db.update_smart(table_name, {"reportState": 1, "phoneState": 49},
                                     condition=f"id={request.task_id}")
            else:
                self.myLog(request.task_city)
                self.db.update_smart(table_name, {"reportState": 1, "phoneState": 1}, condition=f"id={request.task_id}")
        else:
            self.myLog(request.task_city)
            self.db.update_smart(table_name, {"reportState": 1, "phoneState": 404}, condition=f"id={request.task_id}")
            return

        # TODO 4444444444444444444444444444444444444444444444444444444=================================================================

    def error_parase(self, request, response):
        response = self.download(url=request.url, params=request.params)
        if not getattr(response, "json", None):
            yield feapder.Request(
                url=request.url,
                auto_request=False,
                params=request.error_year_params,
                callback=self.error_parase,
                task_digest=request.task_digest,
                task_name=request.task_name
            )
            return
        data = response.json()
        results = data["data"]
        yes = 0
        no = 0
        for data in results:
            if str(data.get('speCause_CN')).startswith('个体工商户未按照《个体工商户年度报告办法'):
                if not data.get('remDate'):
                    _type = 95
                    no = no + 1
                else:
                    _type = 94
                    yes = yes + 1
            else:
                _type = 0
            item = ErrorItem()
            item.abTime = data.get('abntime')  # 列入日期
            item.speCause = data.get('speCause_CN')  # 列入原因
            item.decOrgCN = data.get('decOrg_CN')  # 列入机关
            item.remCause = data.get('remExcpRes_CN')  # 移出原因
            item.remDate = data.get('remDate')  # 移出日期
            item.reDecOrgCN = data.get('reDecOrg_CN')  # 移出机关
            item.digest = request.task_digest
            item.type = _type
            yield item
        log.info(f"{request.task_name} ：共拥有{yes + no}次经营风险:{yes}次解除:{no}次未解除")

    def year_parase(self, request, response):
        response = self.download(url=request.url, params=request.params)
        if not getattr(response, "json", None):
            yield feapder.Request(
                url=request.url,
                auto_request=False,
                params=request.error_year_params,
                callback=self.year_parase,
                task_id=request.task_id,
                task_name=request.task_name,
                task_digest=request.task_digest,
                estDate=request.estDate,
                task_city=request.task_city
            )
        dataList = response.json()
        reportNumberItem = UpdataReportNumberItem()  # base更新差几次年报
        yearList = []
        for data in dataList:  # 入库年报自身信息
            item = YearItem()
            item.year = data.get('anCheYear')
            item.pub_time = data.get('anCheDate')
            item.digest = request.task_digest
            if item.year:
                yearList.append(int(item.year))
                yield item
        if yearList:  # 计算差几次年报
            max_year = max(yearList)  # 最后一次年报的年份
            sum = 2023 - max_year  # 差多少次年报
            reportNumberItem.reportNumber = sum
        else:  # 都没做
            if request.estDate:  # 建立年份
                estDate = datetime.datetime.strptime(request.estDate, "%Y-%m-%d")
                reportNumberItem.reportNumber = int(estDate.year) - 2024
            else:
                reportNumberItem.reportNumber = -1000
        reportNumberItem.id = request.task_id
        yield reportNumberItem
        log.info(f"{request.task_name} : 共做了 {len(dataList)}次年报:还差 {reportNumberItem.reportNumber}次未做")

    def phone_parse(self, request, response):
        response_data = response.json.get('data')
        if not response_data:
            raise ValueError(response.json)

        status = True
        for company in (response_data.get('companyList') or []):
            for i in (company.get('phoneInfoList') or []):
                phone = i.get('number')
                if len(phone) != 11:  # 排除非常规号码
                    continue
                if request.phone == phone and status is True and request.phone:
                    select_number_sql = "select * from business_scope.py_number_segment where number = {}".format(
                        phone[:7])
                    res = self.db.find(select_number_sql, to_json=True) or [{}]
                    res = res[0]
                    log.debug(res)

                    status = False
                    total = response_data.get('companyTotal')
                    item = PhoneItem()
                    item.digest = request.digest
                    item.phoneNum = total
                    item.phone = request.phone
                    item.priori = request.priori
                    item.telType = i.get('type')
                    item.telState = 1
                    item.operator = res.get('operator')
                    item.province = res.get('province')
                    item.city = res.get('city')
                    yield item
                    break

        self.rdc.hincrby(p_count_name, p_cache_success, 1)
        # mysql 需要更新任务状态为做完 即 state=1

    def updata_base(self, re_data: dict, task_id, task_regStatus, task_digest):
        # 去除html标签
        updata_data = {}
        if task_digest:
            updata_data["digest"] = task_digest
        if re_data["entName"]:
            name = re_data["entName"]  # 公司名字 要去掉html标签
            name = MyUtils.remove_html(name)
            name = MyUtils.remove_unchinese(name)
            name = MyUtils.replace_bracket(name)
            updata_data["name"] = name
        if re_data["legelRep"]:
            updata_data["legalName"] = re_data["legelRep"]  # 法人
        if re_data["corpStatus"]:
            updata_data["regStatus"] = re_data["corpStatus"]  # 经营状态编码
        if re_data["corpStatusString"]:
            updata_data["regStatusStr"] = re_data["corpStatusString"]  # 经营描述
        if re_data["entType"]:
            updata_data["entType"] = re_data["entType"]  # 公司类型编码
        if re_data["entTypeString"]:
            updata_data["companyType"] = re_data["entTypeString"]  # 公司类型
        if re_data["regOrg"]:
            updata_data["regInstitute"] = re_data["regOrg"]  # 等级机关
        if re_data["uniscId"]:  # 信用号，要剔除html标签
            creditCode = re_data["uniscId"]
            creditCode = MyUtils.remove_html(creditCode)
            if creditCode:
                updata_data["creditCode"] = creditCode  # 信用号
        if re_data["regNo"]:
            updata_data["regNumber"] = re_data["regNo"]  # 工商号
        # if task_regStatus==1 :                           #旧版数据缺失弥补
        #     if  re_data["corpStatus"]!=1:
        #         updata_data["errorState"]=2
        # else:
        #     if re_data["corpStatus"]==1:
        #         updata_data["errorState"] = 1
        #     else:
        #         updata_data["errorState"] = 2
        if updata_data:
            self.db.update_smart(table_name, updata_data, condition=f"id={task_id}")

    def myLog(self, task_city):
        self.rdc.hincrby("get_base_info", f"{task_city}+{redis_key}", 1)
        log.info(
            f"{task_city}:共计完成 {self.rdc.hget('get_base_info', f'{task_city}+{redis_key}')}次抓取")


redis_key = "齐齐哈尔市:230200：6"  # 第一步
city = 230200  # 第二步
table_name = "py_company_base"
if __name__ == "__main__":
    # 用mysql做任务表，需要先建好任务任务表
    spider = 官渡区(
        redis_key=redis_key,  # 分布式爬虫调度信息存储位置
        task_table=table_name,  # mysql中的任务表
        task_keys=["id", "name", "regNumber", "creditCode", "digest", "city", "provinceId", "district", 'regStatus',
                   'phone'],
        # 需要获取任务表里的字段名，可添加多个
        task_state="reportState",  # mysql中任务状态字段
        # task_condition=f"regStatus in(1,4)  and errorState!=2 and (phone is not null and phone != '' and phone!='-') and `districtId` = '{city}' and entType = 9500  ",
        task_condition=f"regStatus in(1,4)  and (phone is not null and phone != '' and phone!='-') and `cityId` = '{city}' and entType = 9500 ",
        task_limit=1000,
        min_task_count=2000,
        check_task_interval=1000
    )
    # spider.start_monitor_task()  # 第三步
    spider.start()  # 第四步
